// 工具函数

use std::collections::HashMap;

use regex::Regex;
use unicode_segmentation::UnicodeSegmentation;

// 判断文本是否包含中文字符
pub fn is_chinese_text(text: &str) -> bool {
    let chinese_regex = Regex::new(r"[\u4e00-\u9fff]").unwrap();
    chinese_regex.is_match(text)
}

// 简单的中英文分词函数
pub fn tokenize(text: &str) -> Vec<String> {
    let mut tokens = Vec::new();
    
    // 处理中文和英文混合的情况
    for grapheme in text.graphemes(true) {
        // 如果是中文字符，每个字符作为一个token
        if is_chinese_text(grapheme) {
            tokens.push(grapheme.to_string());
        } else {
            // 如果是英文，按空格分词
            let english_words: Vec<&str> = grapheme.split_whitespace().collect();
            for word in english_words {
                if !word.is_empty() {
                    tokens.push(word.to_string());
                }
            }
        }
    }
    
    tokens
}

// 检查是否是常见词
pub fn is_common_word(word: &str) -> bool {
    let common_words = [
        "的", "了", "在", "是", "我", "有", "和", "就", "不", "人", "都", "一", "一个", "上", "也", "很", "到", "说", "要", "去", "你", "会", "着", "没有", "看", "好", "自己", "这",
        "the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "by", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had", "do", "does", "did", "will", "would", "could", "should", "may", "might", "must", "can", "this", "that", "these", "those", "i", "you", "he", "she", "it", "we", "they", "me", "him", "her", "us", "them"
    ];
    
    common_words.contains(&word)
}

// 计算两个符号序列的语义相似度
pub fn calculate_semantic_similarity(
    seq1: &[usize], 
    seq2: &[usize], 
    symbol_relations: &HashMap<usize, f32>
) -> f32 {
    if seq1.is_empty() || seq2.is_empty() {
        return 0.0;
    }
    
    let mut similarity = 0.0;
    let mut count = 0;
    
    for &sym1 in seq1 {
        for &sym2 in seq2 {
            if sym1 == sym2 {
                similarity += 1.0;
                count += 1;
            } else if let Some(&rel) = symbol_relations.get(&sym2) {
                similarity += rel;
                count += 1;
            }
        }
    }
    
    if count == 0 {
        0.0
    } else {
        similarity / count as f32
    }
}